######## Classifier Data Script ########
######## Final Year Project 2011#####
######## Making Web Search Spatial###


######## Liam Sullivan       ########
######## University of Leeds ########

######## Script to pull back classification data for 100 search queries, 
######## each having 100 search results in total

import psyco
psyco.full()


#### Due to Django / Google APp Engine problems need to copy all code rather than importing from Django files

import csv
import urllib
import StringIO


Problems = False

import wsgiref.handlers
import urllib
import logging
import time

from google.appengine.ext import webapp
from google.appengine.ext.webapp import template
from google.appengine.api import urlfetch

from django.utils import simplejson
from operator import itemgetter
from StringIO import StringIO


class SearchResult():
    
    """SearchResult objects have a Title, Extract, URL and Classification data"""
    
    def __init__(self, URL, ClickURL, Title, Extract):
        
        """Build object with passed in values but also generate Classification data from URL and encode in Object"""
        self.URL = URL
        self.ClickURL = ClickURL
        self.Title = Title
        self.Extract = Extract
               
        #Get classification data
        self.Classification = getClassificationData(URL)
        
        #Multiply classification values by 100 to get probabilities as %
        for key, value in self.Classification.iteritems():
            
            ####logging.info("It was: ")
            ####logging.info(value)
            NewValue = value * 100
            
            ####logging.info("Its now: ")
            ####logging.info(NewValue)
            
            NewValue = int(NewValue)
            
            ####logging.info("Finally after rounding it is: ")
            ####logging.info(NewValue)
            
            
            #And alter the value for the key
            self.Classification[key] = NewValue
            
        self.Arts = self.Classification['Arts']
        self.Business = self.Classification['Business']
        self.Science = self.Classification['Science']
        self.Computers = self.Classification['Computers']
        self.Recreation = self.Classification['Recreation']
        self.Sports = self.Classification['Sports']
        self.Society = self.Classification['Society']
        self.Health = self.Classification['Health']
        self.Home = self.Classification['Home']  
        self.Games = self.Classification['Games'] 
        
 
def getSearchResults(appid, query, results, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = ''):
    
    """Get search results, call out to classify each one and build a SearchResult object for each, return a list of these"""
   
    base_url = u'http://search.yahooapis.com/WebSearchService/V1/webSearch?'
    params = locals()
    params['output'] = 'json'
    payload = urllib.urlencode(params)
    url = base_url + payload
    
    responseContent = urllib.urlopen(url)    
    response = StringIO(responseContent.read())


    result = simplejson.load(response)
    
    ProblemsHere = False
    
    try:
        result = result['ResultSet']
    except:
        ProblemsHere = True   
     
    try:
        AllResults = result['Result']
    except:
        ProblemsHere = True
        
        
    ResultObjectList = []
    
    resultCounter = 0
    
    FakedResults = False
    
    print len(AllResults)
    
    for res in AllResults:
        
        if not ProblemsHere:
            
            resultCounter +=1
            
            #Make a new SearchResult Object
            SearchResultInstance = SearchResult(res['Url'], res['ClickUrl'], res['Title'], res['Summary'])
            
            #Use the object method to fetch classification data before saving object
            
            #Add it to the list
            
            ResultObjectList.append(SearchResultInstance)
            print "Got result number: " + str(resultCounter)
            
            
            
        
        else:
            
            FakeResultObjectList = []
            
            FakedResults = True
            
            #Make a fake SearchResult Object
            FakeSearchResultInstance = SearchResult('http://www.google.com', 'http://www.google.com', '9879 Error Fake Data - Ignore Entry', 'A fake entry to avoid halting, ignore this entry as something went wrong')
            #Add it to the list
            ResultObjectList.append(FakeSearchResultInstance)
            
            
            
            
    if FakedResults:
        print "Error - Returned FAKE RESULTS"
        return FakeResultObjectList
    
    else:
        
        print "Returned ALL RESULTS FOR THAT QUERY"
        return ResultObjectList



def getClassificationData(URL):
    """Classifies a URL using the UClassify Service"""
    
    #Dictionary containing all params, which we will encode for REST Call    
    Params = {}   
        
    #BaseURL
    #uClassify URL Endpoint
    endpoint = "http://uclassify.com/browse/uClassify/Topics/ClassifyUrl?"
    
    #API Key    
    Params['readkey'] = "TsE3A35FCBSAdbGFLEvI2PXH0"
    
    #Output Format
    Params['output'] = "json"
    
    #Incoming URL that needs classifying
    Params['url'] = URL
    
    #Version
    Params['version'] = 1.01
    
    #Encode Params from dictionary for call
    
    RestParams = urllib.urlencode(Params)    
    
    #Combine base url and params
    
    RestURL = endpoint + RestParams
    
    time.sleep(0.1)
        
    try:        
        responseContent = urllib.urlopen(RestURL)    
        ClassifierResponse = StringIO(responseContent.read())
        
    except:
        "Failed to get classification"
        
 
    ClassifierResult = simplejson.load(ClassifierResponse)
    
    ##Fake empty results incase don't get any
    
    ResultSet = {'Arts': 0, u'Business': 0, u'Science': 0, u'Computers': 0, u'Recreation': 0, u'Sports': 0, u'Society': 0, u'Health': 0, u'Home': 0, u'Games': 0}
    
    try:
        ResultSet =  ClassifierResult['cls1'] 
        return ResultSet 
    except:
        print "didn't get ANY CLASSIFICATION DATA so pass empty data {}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}"
        Problems = True
        return ResultSet
             
        

#Yahoo APP ID
appid = 'Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-'

### How many search results for each query??     
NumberOfResults = 100
###

#Read in Queries

QueryListFile = open("QueryListFile.txt")

QueryList = []

count = 0

for line in QueryListFile.readlines():
    
    QueryList.append(str(line))
    count += 1
    print "Read query number: " + str(count) + " which is: " + line
    
print "Read in " + str(count) + " queries to pull data for"


#Prepare output file
OutFile = csv.writer(open("ClassifierTestResults.csv", 'w'), dialect='excel')

#Write headings to file followed by new line

Headings = ['Query', 'Result Title','Click URL', 'URL', 'Arts','Games','Home', 'Health', 'Society', 'Sports', 'Recreation','Computers', 'Science']

OutFile.writerow(Headings)

#For each query use existing methods to build a SearchResult object as before

QueryCount = 0

for query in QueryList:
    
    QueryCount += 1
    
    print "Getting search results for query number: " + str(QueryCount)
        
    #Get a list of SearchResult objects
    ResultObjectsList = getSearchResults(appid, query, NumberOfResults, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = '')
    
    ResultCount = 0
    
    for res in ResultObjectsList:
                
        ResultCount ++1       
        
        #For each result output a line with the result details and all classification data
        
        OutLine = [query, res.Title, res.ClickURL, res.URL, res.Arts, res.Games, res.Home,  res.Health, res.Society, res.Sports, res.Recreation, res.Computers, res.Science]
        
        try:
            OutFile.writerow(OutLine)
        except:
            print "ERROR - Skipped a line as couldn't 'print it, seomthing wrong with characters {}{}{}{}{}{"
        
    print "Saved classification data for " + str(ResultCount) + " results from that query..."
    
    if Problems:
        print "But had at least one problem"


print "Done!"
    

